from multiprocessing import Process, Queue
import time
from lxml import etree
import requests
class DouBanSpider(Process):
	def __init__(self, url, q):
		# 重写写父类的__init__方法
		super(DouBanSpider, self).__init__()
		self.url = url
		self.q = q
		self.headers = {
			'Host': 'movie.douban.com',
			'Referer': 'https://movie.douban.com/top250?start=225&filter=',
			'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.104 Safari/537.36',
		}					#请求头部
	def run(self):				#定义run方法
		self.parse_page()		#分析页面
	def send_request(self,url):		#发送请求，返回网页源代码
		i = 0
		while i <= 3:			#请求出错时，重复请求３次,
			try:
				print (u"[INFO]请求url:"+url)
				return requests.get(url=url,headers=self.headers).content
			except Exception as e:
				print(u'[INFO] %s%s'% (e,url))
				i += 1
	def parse_page(self):		#解析网站源码，并采用xpath提取电影名称评分放到队列中
		response = self.send_request(self.url)
		html = etree.HTML(response)
		node_list = html.xpath("//div[@class='info']")	#　获取到一页的电影数据
		for move in node_list:
			title = move.xpath('.//a/span/text()')[0]	# 电影名称
			score = move.xpath('.//div[@class="bd"]//span[@class="rating_num"]/text()')[0]	# 评分
			self.q.put(score + "\t" + title)			#将电影的名称跟评分加入到队列
def main():
	q = Queue()			# 创建一个队列用来保存进程获取到的数据
	base_url = 'https://movie.douban.com/top250?start='		#定义URL
	url_list = [base_url+str(num) for num in range(0,225+1,25)]	# 构造所有URL
	Process_list = []		# 保存进程
	for url in url_list:		# 创建并启动进程
		p = DouBanSpider(url,q)
		p.start()
		Process_list.append(p)
	for i in Process_list:		# 让主进程等待子进程执行完成
		i.join()
	while not q.empty():
		print(q.get())
if __name__=="__main__":
	start = time.time()
	main()
	print('[info]耗时：%s'%(time.time()-start))
